package com.gmss.crawler.service;


import com.alibaba.fastjson.JSON;
import com.gmss.crawler.entity.AcademicianEntity;
import com.gmss.crawler.enums.AcademicianEnum;
import com.gmss.crawler.utils.CommonUtils;
import com.gmss.crawler.utils.CrawlerCommonUtils;
import com.gmss.crawler.utils.FileUtil;
import com.gmss.crawler.utils.RegexUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

/**
 * 中国工程院-院士信息爬取
 */
@Slf4j
@Service
public class CaeCrawlerService {

    @Value("img_file_path")
    private String imgFilePath;

    public final String url = "https://www.cae.cn";
    public final String index = "https://www.cae.cn/cae/html/main/col48/column_48_1.html";
//    public final String index = "https://www.cae.cn/cae/html/main/col323/2019-11/22/20191122095745643268594_1.html";

    public static void main(String[] args) throws IOException {

        Map<String, AcademicianEntity> academicianEntityMap = new HashMap<>();
//        // 2001
//        AcademicianEntity academicianEntity = new AcademicianEntity().builder().name("郑颖人").build();
//        academicianEntityMap.put("郑颖人", academicianEntity);
//        //2001外籍
//        academicianEntity = new AcademicianEntity().builder().name("雅克·康").build();
//        academicianEntityMap.put("雅克·康", academicianEntity);
//        //2003
//        academicianEntity = new AcademicianEntity().builder().name("陈赛娟").build();
//        academicianEntityMap.put("陈赛娟", academicianEntity);
//        //2003外籍
//        academicianEntity = new AcademicianEntity().builder().name("梁基谢夫").build();
//        academicianEntityMap.put("梁基谢夫", academicianEntity);
//        //2005
//        academicianEntity = new AcademicianEntity().builder().name("范本尧").build();
//        academicianEntityMap.put("范本尧", academicianEntity);
//        //2005  大村智
//        academicianEntity = new AcademicianEntity().builder().name("大村智").build();
//        academicianEntityMap.put("大村智", academicianEntity);
//        //2011  张玉卓
//        academicianEntity = new AcademicianEntity().builder().name("张玉卓").build();
//        academicianEntityMap.put("张玉卓", academicianEntity);
//        //2015  李家彪
//        academicianEntity = new AcademicianEntity().builder().name("张玉卓").build();
//        academicianEntityMap.put("张玉卓", academicianEntity);
//        //2017
//        academicianEntity = new AcademicianEntity().builder().name("包振民").build();
//        academicianEntityMap.put("包振民", academicianEntity);
//        academicianEntity = new AcademicianEntity().builder().name("松采夫•康斯坦丁•亚历山德罗维奇").build();
//        academicianEntityMap.put("松采夫•康斯坦丁•亚历山德罗维奇", academicianEntity);
//        //2019
//        academicianEntity = new AcademicianEntity().builder().name("柴立元").build();
//        academicianEntityMap.put("柴立元", academicianEntity);
//        academicianEntity = new AcademicianEntity().builder().name("雷夫·安德森").build();
//        academicianEntityMap.put("雷夫·安德森", academicianEntity);

//        AcademicianEntity academicianEntity = new AcademicianEntity().builder().name("邓文中").build();
//        academicianEntityMap.put("邓文中", academicianEntity);

//        Map<String, String> fileLinesMap = FileUtil.readFileLines("/crawlerUrlFile/cae.txt", ";");
//        new CaeCrawlerService().parseWordData(fileLinesMap, academicianEntityMap, -1);

//        List<AcademicianEntity> list = new CaeCrawlerService().crawlerAca();
//        log.info("{}", JSON.toJSONString(list));
        List<AcademicianEntity> list = new CaeCrawlerService().updateCrawlerAca("2011");
        log.info("{}", JSON.toJSONString(list));
    }

    /**
     * 更新工程院院士人员
     *
     * @param year 当选年
     * @return
     */
    public List<AcademicianEntity> updateCrawlerAca(String year) {
        imgFilePath = imgFilePath + File.separator;

        Document doc = CrawlerCommonUtils.getDocument(index);
        Elements listElements = doc.select(".leftnav_big").select("h3").select("a");
        List<Map<String, String>> acaList = new ArrayList<>();
        for (int i = 0; i < listElements.size(); i++) {
            Map<String, String> map = new HashMap<>();
            String aText = listElements.get(i).text();
            if (StringUtils.isNotBlank(aText)) {
                String aHref = listElements.get(i).attr("href");
                map.put(aText, aHref);
                acaList.add(map);
            }
        }

        Map<String, String> addMap = new HashMap<>();
        if (CollectionUtils.isNotEmpty(acaList)) {
            for (Map<String, String> mapList : acaList) {
                mapList.forEach((key, value) -> {
                    if (StringUtils.equals(key, "历届院士增选情况")) {
                        if (RegexUtils.isHttp(value)) {
                            Document listDetailDoc = CrawlerCommonUtils.getDocument(value);
                        } else {
                            Document listDetailDoc = CrawlerCommonUtils.getDocument(url + value);
                            Elements detailAElements = listDetailDoc.select(".right_block").select(".list2_navm.clearfix").select("ul li a");
                            for (int i = 0; i < detailAElements.size(); i++) {
                                String detailAText = detailAElements.get(i).text();
                                if (StringUtils.isNotBlank(detailAText) && StringUtils.contains(detailAText, year)) {
                                    String detailAHref = detailAElements.get(i).attr("href");
                                    addMap.put(year, detailAHref);
                                }
                            }
                        }
                    }
                });
            }
        }
        Map<String, String> addNameListMap = new HashMap<>();
        if (MapUtils.isNotEmpty(addMap)) {
            addMap.forEach((key, value) -> {
                if (RegexUtils.isHttp(value)) {
                    Document listDetailDoc = CrawlerCommonUtils.getDocument(value);
                } else {
                    Document listDetailDoc = CrawlerCommonUtils.getDocument(url + value);
                    Elements detailAElements = listDetailDoc.select(".right_block").select(".right_md_list2").select("ul li a");
                    for (int i = 0; i < detailAElements.size(); i++) {
                        String detailAText = detailAElements.get(i).text();
                        if (StringUtils.isNotBlank(detailAText) && StringUtils.contains(detailAText, year)
                                && (StringUtils.contains(detailAText, "当选") || StringUtils.contains(detailAText, "增选"))
                                && (StringUtils.contains(detailAText, "名单") || StringUtils.contains(detailAText, "结果"))
                                && !StringUtils.contains(detailAText, "候选人")) {
                            String detailAHref = detailAElements.get(i).attr("href");
                            addNameListMap.put(year, detailAHref);
                        }
                    }
                }
            });
        }

        Map<String, AcademicianEntity> academicianEntityMap = new HashMap<>();
        if (MapUtils.isNotEmpty(addNameListMap)) {
            addNameListMap.forEach((key, value) -> {
                if (RegexUtils.isHttp(value)) {
                    Document listDetailDoc = CrawlerCommonUtils.getDocument(value);
                } else {
                    Document listDetailDoc = CrawlerCommonUtils.getDocument(url + value);
                    Elements zwbgDivElements = listDetailDoc.select(".zwbg_div");
                    for (int i = 0; i < zwbgDivElements.size(); i++) {
                        String departmentText = zwbgDivElements.get(i).select("table").select("caption").select("p").text();
                        departmentText = departmentText.replaceAll("（", "(").replaceAll("）", ")");
                        departmentText = RegexUtils.replaceData(departmentText, RegexUtils.ALL_KUA_REGEX, "");
                        Integer personTitle = null;
                        if (StringUtils.isNotBlank(departmentText)) {
                            personTitle = AcademicianEnum.CAE_ACADEMICIAN.getCode();
                        } else {
                            personTitle = AcademicianEnum.CAE_FOREIGN_ACADEMICIAN.getCode();
                        }
                        Elements tbodyEle = zwbgDivElements.get(i).select("table");
                        CasCrawlerService.getNameListData(academicianEntityMap, tbodyEle, departmentText, year);

                        CasCrawlerService.getTableData(academicianEntityMap, tbodyEle, personTitle);
//                        academicianEntityMap.forEach((name,academiBean)->{
//                            academiBean.setDepartment(departmentNameListMap.get(name));
//                        });

                    }
                }

            });
        }
        log.info("{}年度中国工程院院士增选人数：{}", year, academicianEntityMap.size());
        List<AcademicianEntity> academiList = academicianEntityMap.entrySet().stream().map(entry -> entry.getValue()).collect(Collectors.toList());
        return academiList;
    }


    public List<AcademicianEntity> crawlerAca() {
        imgFilePath = imgFilePath + File.separator;
        Map<String, AcademicianEntity> academicianEntityMap = new HashMap<>();
        Document doc = CrawlerCommonUtils.getDocument(index);
        Elements partElements = doc.select(".right_md_ysmd").select(".ysmd_bt.clearfix");
        AtomicReference<Elements> nameElements = new AtomicReference<>(doc.select(".right_md_ysmd").select(".ysxx_namelist.clearfix ul li"));
        // 院士头衔
//        Integer type = CasCrawlerService.getAcademicianType("中国工程院-全体院士");
        commonData(partElements, nameElements.get(), academicianEntityMap, AcademicianEnum.CAE_ACADEMICIAN.getCode());
        log.info("中国工程院-全体院士:{}", academicianEntityMap.size());
        //存放标题名称和url地址    如：全体院士，外籍院士
        Map<String, String> mapUrl = new HashMap();
        mapUrl.put("中国工程院-全体院士", index);
        List<Map> allUrlList = new ArrayList<>();
        Elements UrlElements = doc.getElementById("li_47_parent_00").select("ul li h3 a");
        for (int i = 0; i < UrlElements.size(); i++) {
            Element element = UrlElements.get(i);
            if (i == 2 || i == 3 || i == 4 || i == 5 || i == 6 || i == 7) {
                mapUrl.put(element.text(), url + element.attr("href"));
            }
        }
        //把存好的名称和url地址添加到list中
        allUrlList.add(mapUrl);
        Map<String, AcademicianEntity> otherMap = new HashMap<>();
        final Integer[] tem = {0};
        for (Map<String, String> map1 : allUrlList) {
            map1.forEach((key, value) -> {
                // key --> 左侧标题名称     value  --> 标题名称的url地址
                if (StringUtils.containsAny(key, "已故", "外籍")) {
                    Document doc1 = CrawlerCommonUtils.getDocument(map1.get(key));
                    nameElements.set(doc1.select(".right_md_ysmd").select(".ysxx_name_tab table tbody tr"));
                    //获取已故和外籍院士的图片
                    Elements tdElements = doc1.select(".right_md_ysmd").select(".ysxx_name_tab table tbody tr td");
                    saveImg(tdElements);
                    // 院士头衔
//                    Integer type2 = CasCrawlerService.getAcademicianType("中国工程院-" + key);
                    Integer type2 = null;
                    if (StringUtils.equals(key, "外籍院士名单")) {
                        type2 = AcademicianEnum.CAE_FOREIGN_ACADEMICIAN.getCode();
                    } else if (StringUtils.equals(key, "已故院士名单")) {
                        // 已故院士名单
                        type2 = AcademicianEnum.CAE_DIE_ACADEMICIAN.getCode();
                    } else if (StringUtils.equals(key, "已故外籍院士名单")) {
                        // 已故外籍院士名单
                        type2 = AcademicianEnum.CAE_DIE_FOREIGN_ACADEMICIAN.getCode();
                    }


                    otherNewData(nameElements.get(), otherMap, type2);

                    academicianEntityMap.putAll(otherMap);
                    log.info("中国工程院-{}:{}", key, otherMap.size() - tem[0]);
                    tem[0] = otherMap.size();
                }
                if (StringUtils.containsAny(key, "香港", "台湾", "资深")) {
                    Document doc1 = CrawlerCommonUtils.getDocument(map1.get(key));
                    nameElements.set(doc1.select(".right_md_ysmd").select(".ysxx_name_tab table tbody tr"));
                    otherAcademicianData(nameElements.get(), academicianEntityMap, null);
//                    academicianEntityMap.putAll(otherMap);
                }
            });
//            academicianEntityMap.putAll(otherMap);
        }

        try {
            Map<String, String> fileLinesMap = FileUtil.readFileLines("/crawlerUrlFile/cae.txt", ";");
            parseWordData(fileLinesMap, academicianEntityMap, null);
        } catch (IOException e) {
            log.error("Exception: {}", e);
        }

        List<AcademicianEntity> list = new ArrayList<>();
        if (MapUtils.isNotEmpty(academicianEntityMap)) {
            academicianEntityMap.forEach((key, value) -> {
                list.add(value);
            });
        }
        return list;
    }

    /**
     * @param nameElements table表信息
     * @param otherMap     院士map-除全本外的
     * @param type         院士头衔
     * @return
     */
    private Map<String, AcademicianEntity> otherNewData(Elements nameElements, Map<String, AcademicianEntity> otherMap, Integer type) {
        Map<String, AcademicianEntity> academicianEntityMap1 = new HashMap<>();
        Map<String, Integer> map = new HashMap<>();
        Elements thead = nameElements.get(0).select("th");
        map = CasCrawlerService.getStringIntegerMap(thead);
//        academicianEntityMap1 = CasCrawlerService.setAcademicianEntityMap(otherMap, map, nameElements, type);
        academicianEntityMap1 = CasCrawlerService.setNewMap(otherMap, map, nameElements, type);
        return academicianEntityMap1;
    }

    private Map<String, AcademicianEntity> otherAcademicianData(Elements nameElements, Map<String, AcademicianEntity> otherMap, Integer type) {
        Map<String, AcademicianEntity> academicianEntityMap1 = new HashMap<>();
        Map<String, Integer> map = new HashMap<>();
        Elements thead = nameElements.get(0).select("th");
        map = CasCrawlerService.getStringIntegerMap(thead);
        academicianEntityMap1 = CasCrawlerService.setAcademicianEntityMap(otherMap, map, nameElements, type);
        return academicianEntityMap1;
    }


    public Map<String, AcademicianEntity> commonData(Elements partElements, Elements nameElements, Map<String, AcademicianEntity> academicianEntityMap, Integer type) {
        List<Map<String, String>> list = new ArrayList<>();
        int partNum = 0;
        for (Element partElement : partElements) {
            String partName = partElement.text();
            //部门人数
            String people = partName.replaceAll("（", "(").substring(partName.indexOf("(") + 1, partName.indexOf("人"));
            for (Element nameElement : nameElements) {
                //取出人名和个人详情地址
                String name = nameElement.select("a").text();
                String detailUrl = url + nameElement.select("a").attr("href");
                Map<String, String> nameMap = new HashMap<>();
                nameMap.put(name, detailUrl);
                list.add(nameMap);
            }
            //截取一个部门的院士
            List<Map<String, String>> list1 = list.subList(partNum, Integer.parseInt(people) + partNum);
            partNum += Integer.parseInt(people);
            Map<String, List<Map<String, String>>> map = new HashMap<>();
            //key--部门名称   value--院士名称和详情地址的map放在list1中
            map.put(partName, list1);
            //根据mapUrl的key值判断对应的类型 例如：全体院士，外籍院士
//            Integer Type = casCrawlerService.getAcademicianType(key);
            //解析数据，下载院士的图片
            Map<String, AcademicianEntity> tmpMap = parseData(map, type);
            if (MapUtils.isNotEmpty(tmpMap)) {
                academicianEntityMap.putAll(tmpMap);
            } else {
                log.debug("tmpMap is null");
            }
        }

        return academicianEntityMap;
    }

    private Map<String, AcademicianEntity> parseData(Map<String, List<Map<String, String>>> map, Integer type) {
        Map<String, AcademicianEntity> academicianEntityMap = new HashMap<>();
        map.forEach((key, value) -> {   //key --> 部门名称    value --> 将Map<姓名，个人详情地址>放入一个List中
            for (Map<String, String> stringStringMap : value) {
                stringStringMap.forEach((name, personUrl) -> {
                    //处理外籍人名
                    name = CommonUtils.handleForeignName(name).replaceAll("（", "(");
                    name = RegexUtils.replaceData(name, RegexUtils.ALL_KUA_REGEX, "");
                    name = RegexUtils.replaceData(name, RegexUtils.CHINA, "");
                    //获取个人详情页的document
                    Document doc = CrawlerCommonUtils.getDocument(personUrl);
                    //获取个人简介
                    Elements desElement = doc.select(".intro p");
                    String describe = "";
                    if (null != desElement && desElement.size() > 0)
                        describe = desElement.text();
                    //详情页中获取到的图片地址
                    String iUrl = doc.select(".info_img > a > img").attr("src");
                    String newName = "";
                    if (StringUtils.isNotBlank(iUrl)) {
                        String picUrl = url + iUrl;
                        //获取图片后缀
                        String postName = picUrl.substring(picUrl.lastIndexOf("."));
                        //给图片重新命名
                        newName = name + postName;
                        //保存图片
                        File file = new File(imgFilePath + newName);
                        if (!file.exists()) {
                            // 处理个人图像中的url包含空格情况
                            picUrl = picUrl.contains(" ") ? picUrl.replaceAll(" ", "%20") : picUrl;
                            CrawlerCommonUtils.getPersonalImgSave(picUrl, imgFilePath, newName);
                        }
//                        log.debug("院士名称：{}....图片名称：{}", name, newName);
                    } else {
                        log.info("没有个人图片");
                    }

                    String key2 = key.replaceAll("一、", "").replaceAll("二、", "").replaceAll("三、", "")
                            .replaceAll("四、", "").replaceAll("五、", "").replaceAll("六、", "")
                            .replaceAll("七、", "").replaceAll("八、", "").replaceAll("九、", "");
                    key2 = RegexUtils.replaceData(key2, RegexUtils.ALL_KUA_REGEX, "");
                    String sex = null;
                    if (StringUtils.contains(describe.replaceAll(" ", "").replaceAll("，", ","), ",男,")) {
                        sex = "男";
                    } else if (StringUtils.contains(describe.replaceAll(" ", "").replaceAll("，", ","), ",女,")) {
                        sex = "女";
                    }
                    //获取院士的信息
                    AcademicianEntity academicianEntity = new AcademicianEntity().builder()
                            .name(name)
                            .imgName(newName)
                            .describeData(describe)   //个人简介
                            .sex(sex)
                            .department(key2)
                            .academicianTitle(type)
                            .academicianTitleValue(AcademicianEnum.getEnum(type).getValue())
                            .build();
                    academicianEntityMap.put(name, academicianEntity);
                });
            }
        });
        return academicianEntityMap;
    }


    private void saveImg(Elements tdElements) {
        List<Map<String, String>> list = new ArrayList<>();
        for (Element nameElement : tdElements) {
            //取出人名和个人详情地址
            String name = nameElement.select("a").text();
            String detailUrl = url + nameElement.select("a").attr("href");
            Map<String, String> nameMap = new HashMap<>();
            if (StringUtils.isNotBlank(name)) {
                nameMap.put(name, detailUrl);
                list.add(nameMap);
            }
        }
        for (Map<String, String> map : list) {
            map.forEach((name, detailUrl) -> {
                //处理外籍人名
                name = CommonUtils.handleForeignName(name).replaceAll("（", "(");
                name = RegexUtils.replaceData(name, RegexUtils.ALL_KUA_REGEX, "");
                name = RegexUtils.replaceData(name, RegexUtils.CHINA, "");
                Document doc = CrawlerCommonUtils.getDocument(detailUrl);
                //获取个人简介
                Elements desElements = doc.select(".intro p");
                String describe = "";
                if (desElements != null && desElements.size() > 0)
                    describe = desElements.text();
                //获取图片地址
                String imgUrl = doc.select(".info_img a img").attr("src");
                String newName = "";
                if (StringUtils.isNotBlank(imgUrl)) {
                    String picUrl = url + imgUrl;
                    //获取图片后缀
                    String postName = picUrl.substring(picUrl.lastIndexOf("."));
                    //给图片重新命名
                    newName = name + postName;
                    //保存图片
                    File file = new File(imgFilePath + newName);
                    if (!file.exists()) {
                        picUrl = picUrl.contains(" ") ? picUrl.replaceAll(" ", "%20") : picUrl;
                        CrawlerCommonUtils.getPersonalImgSave(picUrl, imgFilePath, newName);
                    }
//                    log.debug("院士名称：{}....图片名称：{}", name, newName);
                } else {
                    log.info("没有个人图片");
                }
            });
        }
    }

    public void parseWordData(Map<String, String> fileLinesMap, Map<String, AcademicianEntity> academicianEntityMap, Integer type) {
        for (Map.Entry<String, String> entry : fileLinesMap.entrySet()) {
            String key = entry.getKey();        //年份增选标题
            String value = entry.getValue();        //各个增选年份的地址
            if (StringUtils.containsAny(value, "http")) {
                Document doc = CrawlerCommonUtils.getDocument(value);
                //2001,2013
                if (StringUtils.containsAny(key, "2001", "2013")) {
                    Elements tbodyEle = doc.select("#zoom table tbody");
                    CasCrawlerService.getTableData(academicianEntityMap, tbodyEle, type);
                }
                //2003,2005,2007,2009
                if (StringUtils.containsAny(key, "2003", "2005", "2007")) {
                    Elements tbodyEle = doc.select("#zoom table tbody");
                    Map<String, Integer> tbodyMap = new HashMap<>();
                    for (int i = 0; i < tbodyEle.size(); i++) {
                        Elements trElements = tbodyEle.get(i).select("tr");
                        Elements thead = trElements.get(0).select("td");
                        if (thead.size() == 0) {
                            thead = trElements.get(0).select("th");
                        }
                        tbodyMap = CasCrawlerService.getStringIntegerMap(thead);
                        CasCrawlerService.setAcademicianEntityMap(academicianEntityMap, tbodyMap, trElements, type);
                    }
                }
                //2011,2015
                if (StringUtils.containsAny(key, "2011", "2015")) {
                    Elements table = doc.select("#zoom table");
                    for (int i = 0; i < table.size(); i++) {
                        Elements thead = table.get(i).select("thead tr th");
                        if (thead.size() == 0) {
                            thead = table.get(i).select("thead tr td");
                        }
                        Map<String, Integer> theadMap = CasCrawlerService.getStringIntegerMap(thead);
                        Elements trElements = table.get(i).select("tbody tr");
                        CasCrawlerService.setAcademicianEntityMap(academicianEntityMap, theadMap, trElements, type);
                    }
                }
                //2017,2019
                if (StringUtils.containsAny(key, "2017", "2019")) {
                    Elements tbodyElements = doc.select("#zoom table tbody");
                    for (int i = 0; i < tbodyElements.size(); i++) {
                        Elements trElements = tbodyElements.get(i).select("tr");
                        Elements thead = trElements.get(0).select("td");
                        if (thead.size() < 1) {
                            thead = trElements.get(0).select("th");
                        }
                        Map<String, Integer> tbodyMap = CasCrawlerService.getStringIntegerMap(thead);
                        CasCrawlerService.setAcademicianEntityMap(academicianEntityMap, tbodyMap, trElements, type);
                    }
                }
            }
        }
    }
}
